package com.hubu.search;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class TestCrawler {
    private static String url = "https://cse.seu.edu.cn/";
    private static String path = "articles";
    private static Set<String> url_set = new HashSet<>();
    private static Integer id = 0;

    public static void main(String[] args) {
        getArticleListFromUrl(url);
    }

    /**
      * 获取文章列表
      */
    public static void getArticleListFromUrl(String url) {
        url_set.add(url);
        getArticleFromUrl(url, "计算机科学与工程学院");//先保存本页面的文本
        Document doc = null;
        try {
            doc = Jsoup.connect(url).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
                    .validateTLSCertificates(false).timeout(6000).post();
        } catch (IOException e) {
            e.printStackTrace();
        }
        dealTagLink(doc.getElementsByTag("a")); //找到所有a标签
    }

    private static void dealTagLink(Elements elements) {
        for (Element element : elements) {
            String href = element.attr("href");
            if (href.equals("http://cse.seu.edu.cn/cse_en")) { //不收集英文页面
                continue;
            }
            if (href.contains("_upload") || href.contains("javascript") || href.contains("@") || href.endsWith("http")
                || href.endsWith("https")) {
                continue;
            }
            if (!href.startsWith("http://cse.seu.edu.cn")) {
                if (!url_set.contains(url + href)) {
                    url_set.add(url + href);
                    getArticleFromUrl(url + href, element.text());
                }
            } else if (href.startsWith("/") || href.startsWith("http")){
                if (!url_set.contains(href)) {
                    url_set.add(href);
                    getArticleFromUrl(href, element.text());
                }
            }
        }
    }

    /**
     * 获取文章内容
     * @param detailurl
     * @param title
     */
    public static void getArticleFromUrl(String detailurl, String title) {
        try {
            Document document = Jsoup.connect(detailurl).userAgent("Mozilla/5.0").post();

            dealTagLink(document.select("#container-content a"));//找到所有a标签

            String text = Jsoup.parse(document.select("#container-content").html()).text();
            saveArticle(text, title);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 保存文章到本地
     */
    public static void saveArticle(String content, String title) {
        //文件名中不能有\、/、:、*、?、"、<、>、|
        Pattern pattern = Pattern.compile("[\\s\\\\/:\\*\\?\\\"<>\\|]");
        Matcher matcher = pattern.matcher(title);
        title = matcher.replaceAll(""); // 将匹配到的非法字符以空替换
        if (content.length() == 0) { //内容为空的文章不做存储
            return;
        }

        title = id + "-" + title;
        id++;

        String savePath = path + "/" + title + ".txt";//保存到本地的路径和文件名
        System.out.println(savePath);
        File file = new File(savePath);
        if (file.exists()) {
            return;
        }
        try {
            file.createNewFile();
        } catch (IOException e) {
            e.printStackTrace();
        }

        try {
            FileWriter fw = new FileWriter(file, true);
            BufferedWriter bw = new BufferedWriter(fw);
            bw.write(content);
            bw.flush();
            bw.close();
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
